package org.chen.factory;

import org.chen.processor.AbstractESPNPageProcessor;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.scheduler.QueueScheduler;


public class SpiderFactory {

    public Spider createSpider(String url, PageProcessor pageProcessor, Pipeline pipeline) {
        // 确保完成时退出

        return Spider.create(pageProcessor)
                .addUrl(url)
                .setExitWhenComplete(true)  // 确保完成时退出
                .addPipeline(pipeline)
                .setScheduler(new QueueScheduler())
                .setDownloader(new HttpClientDownloader())
                .thread(1);
    }
}
